from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StringType, IntegerType

if __name__ == '__main__':
    spark = SparkSession.builder. \
        appName("test"). \
        master("local[*]"). \
        getOrCreate()

    sc = spark.sparkContext

    # 基于RDD转换成DataFrame
    rdd = sc.textFile("../data/input/sql/people.txt").map(lambda x: x.split(",")).map(lambda x: (x[0], int(x[1])))
    print(rdd.collect())

    # 构建表机构的描述对象：StructType
    # df = spark.createDataFrame(rdd, schema)
    df1 = rdd.toDF(["name", "age"])
    df1.printSchema()
    df1.show()

    schema = StructType().add("name", StringType(), nullable=True).add("age", IntegerType(), nullable=False)
    df2 = rdd.toDF(schema).coalesce(1)
    df2.printSchema()
    df2.show()

    df2.write.mode("overwrite").parquet("../data/input/sql/people.parquet")
